# -*- coding: utf-8 -*-
'''
Created on Aug 3, 2012

@author: LONG HOANG GIANG
'''
import sys, os
import mechanize
sys.path.append(os.path.expanduser('/home5/vietcntt/longhoanggiang/python'))
sys.path.append(os.path.join(os.path.dirname(__file__), '../'))
import datetime
import re
import simplejson as json
import sqlite3 as db
import gzip
import cStringIO as StringIO

def download(url):
    
    opener = mechanize.build_opener(mechanize.HTTPDefaultErrorHandler, mechanize.HTTPRedirectHandler, mechanize.HTTPRefererProcessor)
    opener.addheaders = [('User-Agent', 'Android App v3.0.3')]
    mechanize.install_opener(opener)
    response = mechanize.urlopen(url)
    data = response.read()
    content_type = response.info().getheader('Content-Type', '')
    if 'x-gzip' in content_type:
        gzipper = gzip.GzipFile(fileobj=StringIO.StringIO(data))
        data = gzipper.read()
    return data


class Wattpad():
    
    storyId = '5030469'
    dbName = 'story'
    
    def __init__(self):
        pass
    
    def getAllChapterId(self, storyId):
        url = 'http://www.wattpad.com/apiv2/info?id={0}&group=1&current_parts='.format(storyId)
        print '> processSaveDb {0}'.format(url)
        res = mechanize.urlopen(url).read()
        data = json.loads(res)
        result = []
        for ichapter in data['group']:
            chapterId = ichapter['ID']
            chapterTitle = ichapter['TITLE']
            chapterTitle = re.sub(r"Chap|Chapter", "Chương", chapterTitle)
            result.append({'id': chapterId, 'title': chapterTitle})
        return result
                
                
    def processSaveDb(self, storyId):
        
        connection = db.connect(self.dbName)
        c2 = connection.c2()
        c2.execute("DROP TABLE IF EXISTS tblChapters")
        c2.execute("DROP TABLE IF EXISTS tblDetail")
        c2.execute("CREATE TABLE tblChapters(`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `title` TEXT)")
        c2.execute("CREATE INDEX chapter_idx ON tblChapters(id)")
        c2.execute("CREATE TABLE tblDetail(`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `chapterId` INTEGER, `detail` TEXT)")
        c2.execute("CREATE INDEX detail_idx ON tblDetail(chapterId)")
        result = self.getAllChapterId(storyId)
        
        for chapterItem in result:
            c2.execute("INSERT INTO tblChapters(title) VALUES('{0}')".format(chapterItem['title']))
            cid = c2.lastrowid
            detail = self.getChapterContent(chapterItem)
            c2.execute('''INSERT INTO tblDetail(chapterId, detail) VALUES({0}, '{1}')'''.format(cid, detail))
        
        c2.close()
        connection.commit()
        connection.close()
            
        
    def getChapterContent(self, chapterItem):
        print '------- processSaveDb {0}: {1}'.format(chapterItem['id'], chapterItem['title'])
        url = 'http://www.wattpad.com/api/text?id={0}&s=0&l=0'.format(chapterItem['id'])
        text = download(url)
        text = re.sub(r"\n{2,}", "<br />", text)
        html = re.sub("\n", '<br />', text)
        html = '''<h4>{0}</h4><br /><br />{1}'''.format(chapterItem['title'], html)
        return html
        
        
        
if __name__ == '__main__':
    
    print '>Start crawler at {0}'.format(datetime.datetime.now())
    
    w = Wattpad()
    w.processSaveDb(Wattpad.storyId)
    
    print '>Finished at {0}'.format(datetime.datetime.now())
    os._exit(1)



    